Initialization



In [413]:

    
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import math
import scipy.io
from scipy.special import expit
from math import *
from scipy import optimize



In [414]:

    
sns.set_style('whitegrid')
%matplotlib inline

Loading Data



In [420]:

    
mat = scipy.io.loadmat('ex3data1.mat')
X = mat['X']
y = mat['y']
X = np.insert(X,0,1,axis=1)

Functions Section



In [674]:

    
#functions Sections
def magic_display(matrix = None):
    if matrix is None:
        # selecting 100 random rows of the X
        rand_indces = np.random.permutation(m)[0:100]
        X_dis = X[rand_indces]
    else:
        X_dis = matrix
      
    if( len(X_dis.shape) > 1 ): 
        m_test,n_test = X_dis.shape
        axis_bound = 1
    else:
        m_test = 1
        n_test = X_dis.shape[0]
        axis_bound = 0
    
    # each number width , height in plot
    example_width = int(round(sqrt(n_test)))
    example_height = int(round( n_test / example_width ))

    # number of numbers to show in plot
    display_rows = floor(sqrt(m_test))
    display_cols = ceil(m_test / display_rows )

    # padding between numbers
    pad = 2

    # intilazation array for holding previos 100 random numbers
    display_array = np.ones((
                             pad + display_rows * ( example_height + pad ),
                             pad + display_cols * ( example_width + pad )
                            ))
    count = 0;
    
    for i in range(display_rows):
        for j in range(display_cols):
            if( count >= m_test ):
                break

            # max_val of each row in X_dis
            max_val = np.max( X_dis[count : count+1], axis= axis_bound)

            # Starting x,y point of numbers shape in array 
            ex_x_range = pad + ( i ) * ( example_height + pad )
            ex_y_range = pad + ( j ) * ( example_width + pad )
            
            if(m_test > 1):
                ex_arr = X_dis[ count : count + 1 , 1:].reshape(example_height , example_width)
            else:
                ex_arr = X_dis[1:].reshape(example_height , example_width)
                
            # Setting values
            display_array[ ex_x_range : ex_x_range + example_height,
                           ex_y_range : ex_y_range + example_width ] = np.divide(ex_arr , max_val)
            count += 1
    
    # Plotting 100 random data
    plt.figure(figsize=(12,8))

    # Get rod of grid
    plt.grid(False)
    plt.imshow(display_array)
    
def compareValueMatrix(i, matrix):
    return np.array([1 if x == iclass else 0 for x in y])

def hyp(theta, X = None):
    if ( X is None ):
        return expit(theta)
    else:
        return expit(np.dot(X,theta))

def cost_function(theta, X, y, _lam):
    J = 0
    
    # finding hypotesis matrix
    h = hyp(theta, X)
    
    # Computing Log(sigmoid(x)) for all of the hypotesis elements
    h1 = np.log(h)
    
    # Computing Log( 1 - simgoid(x)) for all of the hypotesis elements
    h2 = np.log(1 - h)
    
    #Computing Cost of the hypotesis
    J =  ( -1 / m ) * ( np.dot(y.T, h1 ) + np.dot( ( 1 - y).T , h2)) + ( np.dot(theta.T, theta) * _lam / ( 2 * m ))
    
    return J

def gradient_function(theta, X, y, _lam):    
    # finding hypotesis matrix
    h = hyp(theta, X)
    
    # Computing the Gradient Of the Hypotesis
    grad = np.zeros(n).T

    grad[0] = ( 1 / m ) * np.dot(  h - y.T , X[:,0] )
    grad[1:] = ( 1 / m ) * np.dot( h - y.T , X[:,1:] ) + ( _lam / m ) * theta[1:]
    
    return grad

def predict_values(values):
    # theta 10 * 401
    # X 5000 * 401
    
    if( len(values.shape) > 1 ):
        axis_bound = 1
    else:
        axis_bound = 0
        
    return np.argmax(values,axis=axis_bound)

Visualizing Data



In [593]:

    
magic_display()

One vs All Classification



In [500]:

    
m,n = X.shape
label_nums = 10
_lambda = 0
theta = np.zeros(n)
inital_theta = np.zeros(n)
theta_saver = np.zeros((10,n))



In [501]:

    
for i in range(label_nums):
    iclass = i if i else 10
    y_new = compareValueMatrix(iclass, y)
    result = optimize.fmin_bfgs(f= cost_function,x0= inital_theta,fprime= gradient_function, \
                                    args=(X, y_new, _lambda), maxiter=50, \
                                    disp=False,full_output=True)
    theta_saver[i] = result[0]
    print("Cost Function Last value for class " + str(i) + " ==> " + str(result[1]))









    



Cost Function Last value for class 0 ==> 0.015092397366845126
Cost Function Last value for class 1 ==> 0.022409449806085318
Cost Function Last value for class 2 ==> 0.0676947775091393
Cost Function Last value for class 3 ==> 0.07123503680803159
Cost Function Last value for class 4 ==> 0.04939468190383979
Cost Function Last value for class 5 ==> 0.07494862467883623
Cost Function Last value for class 6 ==> 0.03195108104824581
Cost Function Last value for class 7 ==> 0.04470354458476973
Cost Function Last value for class 8 ==> 0.09273787364070607
Cost Function Last value for class 9 ==> 0.08704364051054321

Predecit Values



In [625]:

    
pred = predict_values(np.dot(X, theta_saver.T))
count = 0
for i in pred:
    if( i == 0 ): 
        pred[count] = 10
    count += 1



In [626]:

    
np.average(np.double( y.T == pred)) * 100









    Out[626]:





93.320000000000007



In [529]:

    
# "You should see that the training set accuracy is about 94.9%"
n_correct, n_total = 0., 0.
incorrect_indices = []
for irow in range(X.shape[0]):
    n_total += 1
    if pred[irow] == y[irow]:
        n_correct += 1
    else: incorrect_indices.append(irow)
print("Training set accuracy: %0.1f%%"%(100*(n_correct/n_total)))









    



Training set accuracy: 93.3%



In [590]:

    
# Which Numbers predectid uncorrectly
magic_display(X[incorrect_indices])

Nueral FeedForward



In [618]:

    
#Loading Data
weights = scipy.io.loadmat('ex3weights.mat')
Theta1 = weights['Theta1']
Theta2 = weights['Theta2']



In [670]:

    
def predict(Weight1,Weight2, X):
    hidden_lvl_act = hyp(Weight1.T, X)
    
    if(len(X.shape) > 1):
        axis_bound = 1
    else:
        axis_bound = 0
        
    # Adding columns of 1's to the matrix.
    hidden_lvl_act = np.insert(hidden_lvl_act,0,1,axis=axis_bound)
    
    out_lvl_act = hyp(Weight2.T, hidden_lvl_act)

    return predict_values(out_lvl_act)



In [671]:

    
pred_nueral = predict(Theta1,Theta2,X) + 1



In [672]:

    
np.average(np.double( y.T == pred_nueral)) * 100









    Out[672]:





97.519999999999996



In [ ]:

    
for i in range(1000):
    magic_display(X[i])
    predicted_imgae = predict(Theta1, Theta2, X[i])
    print("This is " + str(predicted_imgae));









    



This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9






    



C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\pyplot.py:524: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)






    



This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 7
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9
This is 9






    



---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-678-4bffe313aef9> in <module>()
      1 for i in range(1000):
----> 2     magic_display(X[i])
      3     predicted_imgae = predict(Theta1, Theta2, X[i])
      4     print("This is " + str(predicted_imgae));

<ipython-input-674-ebd0b41fba57> in magic_display(matrix)
     60 
     61     # Get rod of grid
---> 62     plt.grid(False)
     63     plt.imshow(display_array)
     64 

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\pyplot.py in grid(b, which, axis, **kwargs)
   3788 @docstring.copy_dedent(Axes.grid)
   3789 def grid(b=None, which='major', axis='both', **kwargs):
-> 3790     ret = gca().grid(b=b, which=which, axis=axis, **kwargs)
   3791     return ret
   3792 

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\pyplot.py in gca(**kwargs)
    948     matplotlib.figure.Figure.gca : The figure's gca method.
    949     """
--> 950     return gcf().gca(**kwargs)
    951 
    952 # More ways of creating axes:

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\figure.py in gca(self, **kwargs)
   1367 
   1368         # no axes found, so create one which spans the figure
-> 1369         return self.add_subplot(1, 1, 1, **kwargs)
   1370 
   1371     def sca(self, a):

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\figure.py in add_subplot(self, *args, **kwargs)
   1019                     self._axstack.remove(ax)
   1020 
-> 1021             a = subplot_class_factory(projection_class)(self, *args, **kwargs)
   1022 
   1023         self._axstack.add(key, a)

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\axes\_subplots.py in __init__(self, fig, *args, **kwargs)
     71 
     72         # _axes_class is set in the subplot_class_factory
---> 73         self._axes_class.__init__(self, fig, self.figbox, **kwargs)
     74 
     75     def __reduce__(self):

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in __init__(self, fig, rect, facecolor, frameon, sharex, sharey, label, xscale, yscale, axisbg, **kwargs)
    549 
    550         self._connected = {}  # a dict from events to (id, func)
--> 551         self.cla()
    552         # funcs used to format x and y - fall back on major formatters
    553         self.fmt_xdata = None

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in cla(self)
   1113         self.set_axis_on()
   1114 
-> 1115         self.xaxis.set_clip_path(self.patch)
   1116         self.yaxis.set_clip_path(self.patch)
   1117 

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\axis.py in set_clip_path(self, clippath, transform)
    869         artist.Artist.set_clip_path(self, clippath, transform)
    870         for child in self.majorTicks + self.minorTicks:
--> 871             child.set_clip_path(clippath, transform)
    872         self.stale = True
    873 

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\axis.py in set_clip_path(self, clippath, transform)
    190 
    191     def set_clip_path(self, clippath, transform=None):
--> 192         artist.Artist.set_clip_path(self, clippath, transform)
    193         self.gridline.set_clip_path(clippath, transform)
    194         self.stale = True

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\artist.py in set_clip_path(self, path, transform)
    692             if isinstance(path, Rectangle):
    693                 self.clipbox = TransformedBbox(Bbox.unit(),
--> 694                                                path.get_transform())
    695                 self._clippath = None
    696                 success = True

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\patches.py in get_transform(self)
    219         to the :class:`Patch`.
    220         """
--> 221         return self.get_patch_transform() + artist.Artist.get_transform(self)
    222 
    223     def get_data_transform(self):

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\patches.py in get_patch_transform(self)
    719 
    720     def get_patch_transform(self):
--> 721         self._update_patch_transform()
    722         return self._rect_transform
    723 

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\patches.py in _update_patch_transform(self)
    714         bbox = transforms.Bbox.from_bounds(x, y, width, height)
    715         rot_trans = transforms.Affine2D()
--> 716         rot_trans.rotate_deg_around(x, y, self._angle)
    717         self._rect_transform = transforms.BboxTransformTo(bbox)
    718         self._rect_transform += rot_trans

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\transforms.py in rotate_deg_around(self, x, y, degrees)
   1953         and :meth:`scale`.
   1954         """
-> 1955         return self.translate(-x, -y).rotate_deg(degrees).translate(x, y)
   1956 
   1957     def translate(self, tx, ty):

C:\Users\Danietzio\AppData\Local\Continuum\Anaconda3\lib\site-packages\matplotlib\transforms.py in translate(self, tx, ty)
   1966             [[1.0, 0.0, tx], [0.0, 1.0, ty], [0.0, 0.0, 1.0]],
   1967             np.float_)
-> 1968         self._mtx = np.dot(translate_mtx, self._mtx)
   1969         self.invalidate()
   1970         return self

KeyboardInterrupt:



In [658]:

    
len(X[0:2].shape)









    Out[658]:





2



In [ ]: